from scrapy.selector import HtmlXPathSelector
from scrapy.spider import BaseSpider
from mycrawler.items import CrawlProblemUrl

class Fourthlevel(BaseSpider):

    name = 'CrawlProblem'

    allowed_domin = ['http://www.jyeoo.com/math/ques']

    start_urls = [
        'http://www.jyeoo.com/math/ques/search',

                  ]

    def parse(self, response):

        hxs = HtmlXPathSelector(response)

        sites = hxs.select('//*[@id="page"]/div/table/tbody/tr[2]/td[2]/span')

        items = []

        for site in sites:
            item = CrawlProblemUrl()

            item['url'] = sites.select('a/@href').extract()

            item['kind'] = 4

            items.append(item)

        return  items